
#include "Tokenizer.h"
#include <boost/tokenizer.hpp>


Tokenizer::Tokenizer(std::string& query, std::vector<std::string>& tokens)
{
	typedef boost::tokenizer<boost::char_separator<char> > Tok;
	//boost::char_separator<char> sep;
	//Tok tok(query, sep);
	Tok tok(query);
	for(Tok::iterator tok_iter = tok.begin(); tok_iter != tok.end(); ++tok_iter)
	{
		tokens.push_back(std::string(*tok_iter));
	}
}
